import array, heapq
import os, os.path
assert array.array('f').itemsize == 4
iters = []


def intsfromfile(f):
  while True:
     a = array.array('f')
     a.fromstring(f.read(4000))
     if not a:
         break
     for x in a:
         yield x

'''
print "test in"
inputFileName0 = "/home/diaosi/outputDirForTermScores/soalr_10.txt"
inputFileHandler0 = open(inputFileName0,"r")
for line in inputFileHandler0.readlines():
    print float(line.strip())
print "test out"
'''

basePath = "/home/diaosi/outputDirForTermScores"
letterList = ["0","1","2","3","4","5","6","7","8","9","a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z"]         

numOfBinaryFiles = 0
theBinaryFilesList = []

for eachLetter in letterList:
    theCheckedPath = basePath + "/" + eachLetter
    
    # print theCheckedPath
    '''
    for name in os.listdir(theCheckedPath):
        absoluteFilePathName = theCheckedPath + name
        print absoluteFilePathName
        print os.path.isfile(absoluteFilePathName)
    '''
    
    # how many files are there in total
    for name in os.listdir(theCheckedPath):
        if os.path.isfile(theCheckedPath + "/" + name):
            theBinaryFilesList.append( theCheckedPath + "/" + name )
            numOfBinaryFiles += 1

print "numOfBinaryFiles:",numOfBinaryFiles
print "numOfBinaryFilesInThisList:",len(theBinaryFilesList)

theBinaryFilesList.sort(cmp=None, key=None, reverse=False)

for binaryFileName in theBinaryFilesList[0:625]:
    if binaryFileName.endswith(".binary"):
        print "binaryFileName:",binaryFileName
        currentOutputFileHandler = open(binaryFileName,"rb")
        iters.append( intsfromfile(currentOutputFileHandler) )
    else:
        print binaryFileName
        exit(1)





'''
inputFileName1 = "/home/diaosi/outputDirForTermScores/0/0_8400333.binary" 
inputFileHandler1 = open(inputFileName1,"rb")

inputFileName2 = "/home/diaosi/outputDirForTermScores/1/113_599688.binary" 
inputFileHandler2 = open(inputFileName2,"rb")

inputFileName3 = "/home/diaosi/outputDirForTermScores/2/2_10966214.binary" 
inputFileHandler3 = open(inputFileName3,"rb")

inputFileName4 = "/home/diaosi/outputDirForTermScores/s/snuff_3854.binary" 
inputFileHandler4 = open(inputFileName4,"rb")

inputFileName5 = "/home/diaosi/outputDirForTermScores/s/snyder_47802.binary" 
inputFileHandler5 = open(inputFileName5,"rb")

inputFileName6 = "/home/diaosi/outputDirForTermScores/s/so_3219389.binary"
inputFileHandler6 = open(inputFileName6,"rb")

inputFileName7 = "/home/diaosi/outputDirForTermScores/s/soalr_10.binary"
inputFileHandler7 = open(inputFileName7,"rb")

inputFileName8 = "/home/diaosi/outputDirForTermScores/s/soap_175292.binary"
inputFileHandler8 = open(inputFileName8,"rb")

iters.append(intsfromfile(inputFileHandler1))
iters.append(intsfromfile(inputFileHandler2))
iters.append(intsfromfile(inputFileHandler3))
iters.append(intsfromfile(inputFileHandler4))
iters.append(intsfromfile(inputFileHandler5))
iters.append(intsfromfile(inputFileHandler6))
iters.append(intsfromfile(inputFileHandler7))
iters.append(intsfromfile(inputFileHandler8))
'''

outputFileName = "/home/diaosi/outputDirForTermScores/allScores_sorted.binary"
outputFileHandler = open(outputFileName,"wb")

a = array.array('f')
for x in heapq.merge(*iters):
  a.append(x)
  if len(a) >= 1000:
      a.tofile(outputFileHandler)
      del a[:]
if a:
  a.tofile(outputFileHandler)
  


outputFileHandler.close()
print "done"